In [0]:
################ FINAL PROJECT JAMIE DEANTONIS & RYAN MCNALLY #####################
###### DATASET HOSTED IN GOOGLE DRIVE @ THIS LINK:
###### https://drive.google.com/open?id=1iDNHD1Q0QrQ8EV0MnISyNHvx3bVOeBzs
###### THIS NOTEBOOK WILL RUN END TO END
###### SUMMARY: CLASSES AND METHODS FOR DATA PREPROCESSING AND MODEL BUILDING COME FIRST, SIMPLE EXECUTIONS AT THE BOTTOM
###### FINAL OUTPUT FILE WILL BE WRITTEN FOR YOU AND FILEPATH IS SPECIFIED IN THE FINAL CELL, FEEL FREE TO CHANGE THE FILENAME TO SAVE A NEW FILE TO TEST
In [0]:
# !git clone
# https://gist.github.com/josephernest/3f22c5ed5dabf1815f16efa8fa53d476wav
# !pip install sox
# !sox old.wav -b 16 new.wav
# import josephernest's remake of scipy.io.wavfile with readable 24-bit
# https://gist.github.com/josephernest/3f22c5ed5dabf1815f16efa8fa53d476
# from wav import wavfile as wav_reader
!pip install pysoundfile
!pip install simpleaudio
!pip install playsound
In [0]:
from pathlib import Path # for file handling
from google.colab import drive # for mounting the google drive
import os # for file handling
import scipy.io.wavfile as wf # for reading wav files NO WORK FOR 24BIT
import soundfile # for converting 24 bit to 16 bit
from scipy import signal # for the spectrogram
from scipy import misc
import matplotlib.pyplot as plt # for plotting spectrograms
import numpy as np
import simpleaudio as sa # good for playing the audio directly
from playsound import playsound # also good for playing audio
import tensorflow as tf
from keras.layers import Dense, Dropout, Flatten, Conv2D
from keras.layers import MaxPooling2D, Concatenate
from keras.layers.advanced_activations import LeakyReLU
from keras import Input, Model
import scipy.misc # for writing the fourier transform to file
import tensorflow
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D
from keras.layers import Concatenate
from keras.layers.advanced_activations import LeakyReLU
from keras import Input, Model, Sequential
import shutil # for removing folders
print(tf.__version__)
In [0]:
drive.mount('/content/drive') # mount the google drive
In [0]:
os.chdir('/content/drive/My Drive/Stems') # directory of the stems
data_dir = Path('./Raw') # formalize as a path-like object
print(data_dir)
song_obj
attributes:
stem_list: list of stem objects that fit into that song
audio: an audio object, the audio data for this song
fourier (optional): a fourier object, the fourier transform for the song
functions:
compile_song: gets the audio data
stem_obj
attributes:
filepath: the path to the file containing the stem?
song: the name of the song
stem: the name of the stem
audio: audio object, the audio data for this stem
fourier (optional): a fourier object, the fourier tranform for the stem
functions:
none
audio_obj
attributes:
data: the numpy array of data for the audio
sr: the samplerate of the audio data
functions:
write_audio_to_file: writes audio to file
fourier_obj
attributes:
freqs
times
fourier
sr
audio
functions:
plot: plots the log of the fourier (log is for visual clarity)
fourier_to_audio: converts the fourier and sr to an audio object, initializes the audio attributeIn [0]:
class song_obj: # SONG OBJECT (collection of stems)
def __init__(self, stem_list, name=None, get_fourier=False):
self.stem_list = stem_list
if name != None:
self.name = name
else:
self.name= stem_list[0].song_name
self.audio = self.compile_song() # returns an audio object
if(get_fourier==True):
self.fourier = fourier_obj(self)
else:
self.fourier = None
def compile_song(self):
sr_list = [stem.audio.sr for stem in self.stem_list]
assert(all(x == sr_list[0] for x in sr_list))
# dim1 = 15000000 # just to ensure all files have same shape
data_list = [stem.audio.data for stem in self.stem_list]
# assert(max([data.shape[0] for data in data_list]) < dim1)
# dim2 = max([data.shape[1] for data in data_list])
# audio = np.zeros((dim1))
length = max([data.shape[0] for data in data_list])
audio = np.zeros(length)
for data in data_list:
audio[:data.shape[0]] += data # , :data.shape[1]] += data
return audio_obj(data=audio, sr=sr_list[0])
def get_stem_splits(self):
vocals_stem_list = []
perc_stem_list = []
bass_stem_list = []
other_stem_list = []
for stem in self.stem_list:
stem_group = stem_dict[stem.stem_name]
if stem_group == 'Vocals':
vocals_stem_list.append(stem)
elif stem_group == 'Percussion':
perc_stem_list.append(stem)
elif stem_group == 'Bass':
bass_stem_list.append(stem)
elif stem_group == 'Other':
other_stem_list.append(stem)
else:
print('warning: ', stem.stem_name, 'was not added to any group')
return [song_obj(vocals_stem_list, name='Vocals'),
song_obj(perc_stem_list, name='Percussion'),
song_obj(bass_stem_list, name='Bass'),
song_obj(other_stem_list, name='Other')]
class stem_obj: # STEM OBJECT
def __init__(self, song_name, stem_name, get_fourier=False):
self.filepath = os.path.join('Raw', song_name, stem_name)
self.song_name = song_name
self.stem_name = stem_name
self.audio = audio_obj(filepath=self.filepath)
if(get_fourier==True):
self.fourier = fourier_obj(self.audio)
else:
self.fourier = None
class audio_obj: # AUDIO OBJECT
def __init__(self, data=None, sr=None, filepath=-1):
# one of (audio, sr) or (filepath) is required
# this is often the cause of problems if wrong if statement entered
if filepath != -1:
self.data, self.sr = soundfile.read(filepath)
if len(self.data.shape) >= 2: # stereo
self.data = self.data[:, 0] # need to think about stereo handling
# self.data = np.mean(data, axis=1) # i think this is correct
else:
assert isinstance(data, np.ndarray), "specify filepath or data"
self.data = data
self.sr = sr
if isinstance(data, tuple):
self.data = self.data[0]
assert self.data.ndim == 1, "audio data must be 1d"
def to_fourier(self): # convert to fourier
self.fourier = fourier_obj(self)
return self.fourier
def to_file(self, filepath): # save as .wav
soundfile.write(filepath, self.data, self.sr)
def boxplot(self): # boxplot of the spectrum of values in this audio
plt.boxplot(self.data)
class fourier_obj: # FOURIER OBJECT
def __init__(self, audio=-1, # input is either this line
fourier=-1, freqs=-1, t=-1, sr=44100, # or this line
abs=False, get_2d=True, get_ri=True):
# freqs : frequencies : freqs corresponding to 1st axis of fourier
# t : times : times corresponding to 2nd axis of fourier
# fourier : fourier series : 2d np array of fourier
assertion = not isinstance(audio, int) or not isinstance(fourier, int)
assert assertion, "must either specify audio_obj or fourier array"
if isinstance(fourier, int):
self.m = 1024
[self.freqs, self.t, fourier] = signal.stft(audio.data,
nperseg=self.m,
noverlap=3*self.m//4)
self.sr = sr
self.fourier_ri = self.get_two_channel_fourier(fourier)
else: # if just passing in a fourier to make the object
if fourier.ndim == 2:
fourier = fourier
self.m = fourier.shape[0] * 2 - 2
self.freqs = freqs
self.t = t
self.sr = sr
self.fourier_ri = self.get_two_channel_fourier(fourier)
else: # the fourier defined is actually the fourier_ri
assert fourier.ndim == 3, "fourier_obj must have ndim 2 or 3"
self.fourier_ri = fourier
# self.fourier = self.ri_to_complex()
self.m = self.fourier_ri.shape[0] * 2 - 2
self.freqs = freqs
self.t = t
self.sr = sr
if abs == True:
self.fourier = np.abs(self.fourier)
assert self.fourier_ri.ndim == 3, "fourier ri must be 3d"
def get_two_channel_fourier(self, fourier):
# for real in first channel, imaginary in second
f = np.zeros(fourier.shape + tuple((2,)))
f[:, :, 0] = np.real(fourier)
f[:, :, 1] = np.imag(fourier)
return f
def ri_to_complex(self):
# convert the 2-channel fourier to a standard 2d fourier
real = np.array(self.fourier_ri[:, :, 0], dtype=complex)
imag = np.array(self.fourier_ri[:, :, 1], dtype=complex) * 1j
return real + imag
def to_audio(self): # convert to audio
fourier = self.ri_to_complex()
ts, data = signal.istft(fourier, self.sr,
nperseg=self.m, noverlap=3*self.m//4)
self.audio = audio_obj(data=data, sr=self.sr)
return self.audio
def to_file(self, songname, filename): # save as jpg
path = os.path.join('Train', songname, 'TRANSFORM', filename + '.jpg')
scipy.misc.imsave(path, self.fourier)
def plot(self):
# log makes the plot more readable
fourier_like = np.log(np.abs(self.fourier).clip(min=0.0000000001))
plt.pcolormesh(self.t, self.freqs, fourier_like)
plt.ylabel('Frequency [Hz]')
plt.xlabel('Time [sec]')
plt.show()
In [0]:
stem_dict_init = {
'Vocals' : ['BG Vox', 'Lead Vox', 'Vox Chop', 'Vocals',
'Ahh Stem', 'Drew Ad Lib Stem', 'Drew LD Vox Stem',
'Emily Vox Stem', 'Gang Vocal Stem', 'Hook Stem',
'Vocoder Stem', 'Vox Chop Stem', 'Vox Pad Stem',
'Vocal Filter', 'Vocal Glitch'],
'Percussion' : ['Drums', 'Claps', 'Hi Hats', 'Snares', 'Crash Stem',
'Hi Hats Stem', 'Kick Stem', 'SFX Stem', 'Snare Stem',
'Toms Stem', 'Clap', 'Hi Hats & Cymbals', 'Impacts',
'Kick', 'Live Drums', 'Percussion', 'Toms'],
'Bass' : ['Bass', '808', 'Bass Stem', 'Wide Bass Stem',
'Live Bass'],
'Other' : ['Glitch Synth', 'GTR', 'Piano', 'Sax', 'SFX',
'Strings & Slide GTR', 'Synth Chords',
'Guitars', 'Risers', 'Synths', 'Arp Stem',
'BGV Stem', 'GTR Stem', 'Horns Stem', 'Keys Stem',
'Pad Stem', 'Piano Stem', 'Pluck Stem',
'Sax Pad Stem', 'Sax Stem', 'Strings Stem',
'Acoustic Guitar', 'Arp Synth', 'Drop Layers',
'Drop Synth 2', 'E Guitar 2', 'Fills', 'Final Synth',
'Piano 1', 'Piano 2', 'Wurli']
}
stem_dict = {} #
for group in stem_dict_init.keys():
for stem in stem_dict_init[group]:
stem_dict[stem + '.wav'] = group
In [0]:
def compile_tracks_and_stems():
# loops through each song, makes (1) the full track file
# (2) the vocal, perc, bass, other tracks
for song in os.listdir(data_dir): # loop through each song
print('*****' + song)
stems = [] # intialize what will be a nested list of stem objects
# (1) make the full track file
for stem in os.listdir(os.path.join(data_dir, song)): # stems in song
# print('***** ', song, '***** ', stem)
temp = stem_obj(song, stem, get_fourier=False)
stems.append(temp) # add current stem to the list
song = song_obj(stem_list=stems)
song.audio.write_to_file(os.path.join('Train', # full song file
song.name,
'FULL_TRACK/song.wav'))
# (2) make the vocal, perc, bass, other tracks
for stem_split in song.get_stem_splits():
# write the full stems into files
stem_split.audio.write_to_file(os.path.join('Train', # stemgroup file
song.name,
'STEMS',
stem_split.name + '.wav'))
# compile_tracks_and_stems()
In [0]:
class koretzky_obj:
def __init__(self, stem_name, n, is_regression=False):
self.me = model_essentials(n=n, is_regression=is_regression)
self.stem_name = stem_name
first = True
mew = True
for i, song in enumerate(os.listdir(data_dir)):
if song == 'DLMD':
continue
if first == True: # first iteration only
track = koretzky_track_obj(song, stem_name, self.me)
inputs = track.full_track.inputs
targets = track.stem.outputs
first = False
del track
break
continue
# second iteration and beyond:
inputs = np.concatenate((inputs,
self.get_inputs(song_name=song, mew=mew)))
targets = np.concatenate((targets,
self.get_targets(song_name=song,
stem_name='Percussion',
is_reg=self.me.is_regression,
mew=mew)))
if i >= 2:
break
self.inputs = inputs
self.targets = targets
self.model = self.koretzky()
def koretzky(self):
shape = (self.me.m, self.me.n, 2)
input_shape = Input(shape=shape, name='input')
# # concatenated_stems = []
for i in range(1):
i = str(i + 1)
x = Conv2D(32, (3,3), padding='same',input_shape=shape)(input_shape)
x = LeakyReLU(name='leaky_' + i + '1')(x)
x = Conv2D(16, (3, 3), padding='same', name='conv16_' + i + '1')(x)
x = LeakyReLU(name='leaky_' + i + '2')(x)
x = MaxPooling2D(pool_size=(3, 3), name='max33_' + i + '1')(x)
x = Dropout(0.25, name='dropout25_' + i + '1')(x)
x = Conv2D(64, (3, 3), padding='same', name='conv64_' + i)(x)
x = LeakyReLU(name='leaky_' + i + '3')(x)
x = Conv2D(16, (3, 3), padding='same', name='conv16_' + i + '2')(x)
x = LeakyReLU(name='leaky_' + i + '4')(x)
x = MaxPooling2D(pool_size=(3,3), name='max33_' + i + '2')(x)
x = Dropout(0.25, name='dropout25_' + i + '2')(x)
x = Flatten(name='flatten_' + i)(x)
x = Dense(128, name='dense128_' + i)(x)
x = LeakyReLU(name='leaky_' + i + '5')(x)
x = Dropout(0.5, name='dropout50_' + i)(x)
x = Dense(self.me.m * 2, activation=self.me.activation,
name='dense513_' + i)(x)
# # concatenated_stems.append(x)
# concatenate output layers
# # merged_output=Concatenate(axis=1,name='concat')(concatenated_stems)
# connect to single/common input layer
# # model = Model(input_shape, merged_output)
model = Model(input_shape, x)
model.compile(optimizer=self.me.opt,
loss=self.me.loss,
metrics=['accuracy'])
return model
def predict_track(self, song): # koretzky_track_obj
# input_obj = koretzky_input_obj(song, self.me)
input_slices = self.get_inputs(song) # input_obj.slices.slices # a numpy array
print('got input slices')
fourier = fourier_slices_obj(input_slices).to_fourier(44100).fourier_ri # input_obj.fourier_obj.fourier_ri
sr = 44100 # input_obj.sr
# del input_obj # delete the object entirely
print('got fourier_slices')
preds = self.model.predict(input_slices)
print('got preds')
return koretzky_prediction_obj(track=fourier, sr=sr,
model_input=input_slices,
model_output=preds,
song=song,
stem=self.stem_name,
me=self.me)
def output_slices_to_audio(self):
x = np.repeat(np.expand_dims(self.outputs[0][0], axis=2),
self.n, axis=2)
y = np.reshape(x, (x.shape[0], x.shape[1] * x.shape[2]))
print(y.shape)
audio = fourier_obj(fourier=y, sr=self.srs[0]).to_audio()
audio.write_to_file(os.path.join('test_files', 'sliced_up2.wav'))
print(audio.data.shape)
print('file saved in test_files/sliced_up2.wav')
return(audio)
def get_inputs(self, song_name, mew=False):
# turn the fourier into slices (either with overlapping window
# or non-overlapping window (mutually_exclusive_windows))
path = os.path.join('Train', song_name, 'FULL_TRACK', 'song.wav')
f = fourier_obj(audio_obj(filepath=path)).fourier_ri
n = self.me.n
shape = f.shape
shape1 = shape[1] // n
fs = f[:, :(n * shape1), :] # clip for even division
# fs = np.real(fs) # HOW TO HANDLE COMPLEX VALUES????
if mew == False:
temp_1 = np.zeros((fs.shape[0], fs.shape[1] + n - 1, 2)) # padding
temp_1[:, (n-1)//2 : temp_1.shape[1] - (n-1)//2, :] += fs
temp_2 = np.zeros((fs.shape[1], fs.shape[0], n, 2))
for i in range(temp_2.shape[0]):
temp_2[i, :, :, :] += temp_1[:, i : i+n, :] # (i, freq, t, 2)
else: # mutually_exclusive_windows == True
temp_1 = np.transpose(fs, (1, 0, 2))
temp_2 = np.reshape(temp_1, (temp_1.shape[0] // n,
n, temp_1.shape[1], 2))
assert temp_2.ndim == 4, "temp_2 must be 4d"
return temp_2
def get_targets(self, song_name, stem_name, is_reg=False, mew=False):
path = os.path.join('Train', song_name, 'STEMS', stem_name + '.wav')
f = fourier_obj(audio_obj(filepath=path)).fourier_ri
if is_reg == False: # if using a mask
mask_thresh = np.percentile(f.flatten(), 97) # 97th pct threshold
f = np.where(f > mask_thresh, 1, 0)
n = self.me.n
shape = f.shape
shape1 = shape[1] // n
fs = f[:, :(n * shape1), :] # clips for even division
fs = np.transpose(fs, (1, 0, 2)) # first axis should be timesteps
if mew == True:
fs = np.reshape(fs, (n, shape1, fs.shape[1], 2)) # (n, shp1, m, 2)
fs = fs[(n - 1) // 2, :, :, :] # (shape1, m) (dim1 vanishes?)
assert fs.ndim == 3, "fs must be 3d"
shape = fs.shape
return np.reshape(fs, (shape[0], shape[1] * shape[2]))
class model_essentials: # model essential values
def __init__(self, is_regression=-1, n=-1):
self.is_regression = is_regression
self.n = n # the number of timesteps in each slice
self.m = 513 # the first axis of fourier
if is_regression == True:
self.loss = 'mse'
self.opt = 'adam'
self.activation = 'linear'
else:
self.loss = 'binary_crossentropy'
self.opt = 'adam'
self.activation = 'sigmoid'
In [0]:
############ TRACK OBJECTS
class koretzky_track_obj:
def __init__(self, song_name, stem, me):
assert me.n % 2 == 1, "n must be an odd integer"
self.full_track = koretzky_input_obj(song_name, me, mew=False)
# self.vocals = koretzky_target_obj(song_name, 'Vocals',
# me, self.full_track)
# self.bass = koretzky_target_obj(song_name, 'Bass',
# me, self.full_track)
self.stem = koretzky_target_obj(song_name, stem,
me, mew=False)
# self.other = koretzky_target_obj(song_name, 'Other',
# me, self.full_track)
self.sr = self.full_track.sr
class koretzky_input_obj:
def __init__(self, song_name, me, mew=False):
filepath = self.get_path(song_name)
assert me.n % 2 == 1, "n must be an odd integer"
self.n = me.n # the number of samples in each split (25 per koretzky)
self.track = audio_obj(filepath=filepath) # start with an audio obj
self.sr = self.track.sr # follows from the audio obj
self.fourier_obj = fourier_obj(audio=self.track)
self.slices = self.get_slices_obj(mew) # take slices of the raw track
data = self.slices.slices
#r = np.real(data)
#c = np.imag(data)
self.inputs = data #tf.complex(r, c)
def get_path(self, song_name):
return os.path.join('Train', song_name, 'FULL_TRACK', 'song.wav')
def apply_mask(self, mask):
f = self.fourier_obj
masked = f.fourier_ri * mask
f1 = fourier_obj(fourier=masked, freqs=f.freqs, t=f.t, sr=f.sr)
return f1
def get_slices_obj(self, mutually_exclusive_windows=False):
# turn the fourier into slices (either with overlapping window
# or non-overlapping window (mutuall_exclusive_windows))
f = self.fourier_obj
n = self.n
shape = f.fourier_ri.shape
shape1 = shape[1] // n
fs = f.fourier_ri[:, :(n * shape1), :] # clip for even division
# fs = np.real(fs) # HOW TO HANDLE COMPLEX VALUES????
if mutually_exclusive_windows == False:
temp_1 = np.zeros((fs.shape[0], fs.shape[1] + n - 1, 2)) # padding
temp_1[:, (n-1)//2 : temp_1.shape[1] - (n-1)//2, :] += fs
temp_2 = np.zeros((fs.shape[1], fs.shape[0], n, 2))
for i in range(temp_2.shape[0]):
temp_2[i, :, :, :] += temp_1[:, i : i+n, :] # (i, freq, t, 2)
else: # mutually_exclusive_windows == True
temp_1 = np.transpose(fs, (1, 0, 2))
temp_2 = np.reshape(temp_1, (temp_1.shape[0] // n,
n, temp_1.shape[1], 2))
assert temp_2.ndim == 4, "temp_2 must be 4d"
return fourier_slices_obj(temp_2)
class koretzky_target_obj:
def __init__(self, song_name, stem, me, mew=False):
assert me.n % 2 == 1, "n must be an odd integer"
filepath = self.get_path(song_name, stem)
self.n = me.n
self.track = audio_obj(filepath=filepath) # start with an audio obj
self.sr = self.track.sr # follows from the audio obj
if me.is_regression:
self.fourier_obj = self.get_mask_from_division()
else:
self.fourier_obj = self.get_mask_from_pct(97)
self.slices = self.get_slices_obj(mew)
shape = self.slices.slices.shape
self.outputs = self.slices.slices.reshape((shape[0],
shape[1] * 2))
def get_path(self, song_name, stem):
return os.path.join('Train', song_name, 'STEMS', stem + '.wav')
def get_mask_from_pct(self, pct):
f = fourier_obj(audio=self.track)
mask_thresh = np.percentile(f.fourier_ri.flatten(), pct)
mask = np.where(f.fourier_ri > mask_thresh, 1, 0)
return fourier_obj(fourier=mask, freqs=f.freqs, t=f.t, sr=f.sr)
def get_mask_from_division(self):#, full_song):
#f = fourier_obj(audio=self.track)
#mask = f.fourier / np.where(full_song.fourier_obj.fourier == 0,
# 1e-9, full_song.fourier_obj.fourier)
# return fourier_obj(fourier=mask, freqs=f.freqs, t=f.t, sr=f.sr)
return fourier_obj(audio=self.track) # let's just predict the stem
def get_slices_obj(self, mutually_exclusive_windows=False):
f = self.fourier_obj
n = self.n
shape = f.fourier_ri.shape
shape1 = shape[1] // n
fs = f.fourier_ri[:, :(n * shape1), :] # clips for even division
fs = np.transpose(fs, (1, 0, 2)) # first axis should be timesteps
if mutually_exclusive_windows == True:
fs = np.reshape(fs, (n, shape1, fs.shape[1], 2)) # (n, shp1, m, 2)
fs = fs[(n - 1) // 2, :, :, :] # (shape1, m) (dim1 vanishes?)
assert fs.ndim == 3, "fs must be 3d"
return fourier_slices_obj(fs)
class koretzky_prediction_obj:
def __init__(self, track, sr, model_input, model_output,
song, stem, me):
# self.model_input = model_input
self.song = song
self.stem_name = stem
self.model_output = model_output # np array of mask pred
# self.track = track # koretzky track object of actual track
target_ri = koretzky_target_obj(song_name=song,
stem=stem,
me=me).fourier_obj.fourier_ri
slices = fourier_slices_obj(self.model_output) # slices obj of MO
fourier = slices.to_fourier(sr=sr) # fourier obj of MO
self.sr = sr # sr
self.vox_preds = fourier.fourier_ri # fourier ri data of MO
if me.is_regression == False:
self.target_audio = self.apply_mask(target_ri,
track)
self.predicted_audio = self.apply_mask(self.vox_preds, track)
else: # regression == True
temp = fourier_slices_obj(model_input)
#self.input_audio = temp.to_fourier(sr=sr).fourier_ri
#targets = fourier_slices_obj(self.target.outputs)
self.target_audio = target_ri# .to_fourier(sr=sr).fourier_ri # np array
self.predicted_audio = self.vox_preds
def apply_mask(self, target, track):
vox = np.zeros(track.shape)
vox[:min(vox.shape[0], target.shape[0]),
:min(vox.shape[1], target.shape[1]),
:] = target[:min(vox.shape[0], target.shape[0]),
:min(vox.shape[1], target.shape[1]), :]
return track * vox
def to_folder(self, folder_path):
path = os.path.join('test_files', self.song, folder_path)
if os.path.exists(path):
print('path already exists - deleting the old one')
shutil.rmtree(path)
os.mkdir(path)
# write the input to audio
# audio = fourier_obj(fourier=self.input_audio,
# sr=self.sr).to_audio()
# audio.to_file(os.path.join(path, 'input.wav'))
# write the prediction to audio
audio = fourier_obj(fourier=self.predicted_audio,
sr=self.sr).to_audio()
audio.to_file(os.path.join(path, self.stem_name + '.wav'))
del audio
# write the output to audio
audio = fourier_obj(fourier=self.target_audio,
sr=self.sr).to_audio()
audio.to_file(filepath=os.path.join(path, 'actual.wav'))
class fourier_slices_obj:
def __init__(self, slice_array):#, ts_array, freqs_array):
if slice_array.ndim == 2: # need to unflatten the output
shape = slice_array.shape
slice_array = np.reshape(slice_array, (shape[0],
shape[1] // 2, 2))
fs = slice_array
m = slice_array.shape[0]
self.slices = slice_array
# self.slice_objs = [fourier_obj(fourier=fs[i, :]) for i in range(m)]
def to_fourier(self, sr): # convert back to fourier
slices = np.array(self.slices)
if len(slices.shape) != 3: # (m, freq_range, len_slice, channels)
assert len(slices.shape) == 4, "must have shape len 3 or 4"
mid = (slices.shape[2] - 1) // 2
slices = slices[:, :, mid, :] # should turn into (m, freq_range, 2)
# slices are singleton columns of fourier (m, freq_range)
fourier_ri = np.transpose(slices, (1, 0, 2))
print('creating fourier object')
return fourier_obj(fourier=fourier_ri, sr=sr) # will recognize ri
Space below to implement
In [0]:
# loads the tracks, gets the fouriers, apply masks, make slices
n = 25 # width of each slice
k = koretzky_obj('Percussion', n, is_regression=False)
In [0]:
# use slices as target, fit the model
# slightly worse now cause predicts for both channels separately
k.model.fit(x=k.inputs, y=k.targets, epochs=5)
del k.inputs
del k.targets
In [0]:
# predict slices on a song
pred_obj = k.predict_track('CLOSER') # RAM spikes and then goes down
In [0]:
# prediction is saved to /content/drive/My Drive/Stems/test_files/[song_name]/actual.wav
pred_obj.to_folder('test_12_20_331_regression_vRM')
In [0]: